{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "3PgIns-jRj_8",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 437.0
    },
    "outputId": "3956131d-9474-4ca2-bb14-19ec689ad61c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1000 cost = 2.263604\n",
      "Epoch: 2000 cost = 2.092835\n",
      "Epoch: 3000 cost = 1.889603\n",
      "Epoch: 4000 cost = 2.024243\n",
      "Epoch: 5000 cost = 1.588786\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd8AAAFKCAYAAABcq1WoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8U1X+//FXlja0dA1NVUBUwAKi\nIPxQBwXRWhA3ZC+i+HVEhtEBh1FRNgdQQcERNxSwqCAoVBYVRYZ9V2QYEAQdi7sC2pYmaWvbtFl+\nfzATh7GA1vamSd7Px8OH3Jzk3s85lLx7zz25MQUCgQAiIiJiGHOoCxAREYk2Cl8RERGDKXxFREQM\npvAVERExmMJXRETEYApfERERg1mNOlBBQYlRhzJcamo8TmdZqMuIGhpv42nMjaXxNlZdjbfDkXjC\nNp351gKr1RLqEqKKxtt4GnNjabyNFYrxVviKiIgYTOErIvXOkSOH6dbtkl/1mm+++YoPP9xdRxWJ\n1C6Fr4hEhM2bNyl8JWwYtuBKROTXeuedt1iyZBElJSXceedIrrqqB08++Ti7dn2A1+ulXbv2jB07\nkR073mPhwpeJiYmhpKSEkSP/wltvLSc391UqKytp2/YCxo37KzZbg1B3SQRQ+IpIPeX3+/F6q5g/\nfzEbN65j1qxniYmJYd++PSxY8Do+n4+hQ29h/fo1XH31tVx++ZU0adKU2267g7179zB37mxefvlV\n0tIcPP74VHJyZjNixKhQd0sE0LSziNRTgUCAnj2vByAjozUFBflcccVVzJ27AKvVis1mo3Xr8zh8\n+NDPXrt9+xauuqo7aWkOAHr37sfmzRsNrV/kZHTmKyL1ksVioUGDY9PEZrMZv9+P0+nkqaem8+mn\nn2I2mygqOsqAATf97LUlJaVs3bqRnTt3AOD3B/B6qwytX+RkFL4iEjZeeOF5rFYrr7yymNjYWCZP\nnlDt89LS0ujZ83pNM0u9pWlnEQkbLlcRzZu3JDY2loMH8/joo72Ul5cDYLVaKS0tBaBLl25s3rwR\np9MJwNatm1i4cF6oyhb5GZ35ikjYGDToFh55ZBLvvvs27dp1YMSIUTz22MOcd975XHZZVyZPnsD3\n3x/mkUemc+utv2fkyOEEAn5SU+2MHj0u1OWLBJkCgUDAiANF8r2dHY7EiO5ffaPxNp7G3Fgab2PV\n1Xjr3s4iEnX8/iqqPEX4/VpoJfWPpp1FJKIEAn6ch9ZQ7voUX5UbS0wycSmtSG3SA5NJ5xtSPyh8\nRSSiOA+tobRgZ3DbV+UObtub9gxVWSLH0a+BIhIx/P4qyl2fVttW7srTFLTUGwpfEYkYvqoSfFXu\nE7S58VVpEZPUDwpfEYkYlphELDHJJ2hLxhJz4tWnIkZS+IpIxDCbY4hLaVVtW1xKBmZzjMEViVRP\nC65EJKKkNukBHLvG+9Nq54zg4yL1gcJXRCKKyWTG3rQn/sZX4asqwRKTqDNeqXc07SwSIR5++K9s\n27alVvb17rtv8+c/31Ur+woVszmGGJtdwSv1ks58RSLEgw8+FOoSROQXUviK1FNvv/0mixcvxOfz\n0ahRGg8++BC7d+/ivfe20bBhQ/bu/RCr1cJDDz1G8+YtGDHiD9xwQ2+uvvpaunTpxP33j2fp0sWU\nlpYyfvwkVqx4g/3793H22c2ZNm0GVquVbds288ILz1NVVUVcXDxjxz7IuedWv2BJRGqPpp1F6iGn\ns4gnn5zOk08+x+LFb9CkSVPmzZsLwI4d2+nTZwCLFy+nQ4dOLFmyqNp9uN0uXnkll8zM7owffz9D\nhw5n0aLlfPHFZ3z44W68Xi+PPDKJ+++fwKJFy+natRszZz5tXCdFopjCV6QeSk21s3r1ZtLTTwOg\nffsOHD58CICzz25O69ZtAGjVqhU//PB9tfvo2vUKAFq0aEmTJk1p1uwsYmNjadr0TAoLC7Barbzz\nzlrOP/+Cnx1DROqWpp1F6iGfz8fcubPZvn0LPp+PsrIyzjyzGQANGyYEn2c2W/D5/NXuIz4+/t/P\nMRMXF3fca/z+Y69ZsmQxf//7O1RWVlJZWYnJZKqrLonIf1H4itRD69evZfv2LcycmUNKSgorVrzB\nmjWravUYH320l1dfnU9OznzOOKMx//jHDqZNm1KrxxCR6mnaWaQecrmKOP30M0hJScHtdrFhw1rK\ny8tr9RhOp5PU1FROO+10KioqWLVqJRUV5QQCgVo9joj8nMJXpB7Kyroat9tNdnZvJk0az7Bhd5Gf\n/wMzZz5Va8e45JLOpKU5yM7uzV/+8icGDryJhg0TmDDh/lo7hohUzxQw6NfcgoLI/TYRhyMxovtX\n32i860ZVlY+y0kriE2KJibEc16YxN5bG21h1Nd4Ox4m/yEPXfEWinN/v570Nn/NlXiGlxR4Skmyc\nk5HGpZktMJs1OSZSFxS+IlHuvQ2f89Gunz5iVFrsCW53yTo3VGWJRDT9WisSxaqqfHyZV1ht21d5\nhVRV+QyuSCQ6KHxFolhZaSWlxZ5q20pLPJSVVhpckUh0+E3TztOnT+ef//wnXq+X4cOH06OHvi9T\nJJzEJ8SSkGSrNoATEm3EJ8SGoCqRyFfj8N2xYwcHDx4kNzcXp9NJnz59FL4iYSYmxsI5GWnHXfP9\nj7Mz0n626llEakeNw/eiiy6iXbt2ACQlJVFeXo7P58Ni0T9WkXByaWYL4Ng13tISDwmJNs7+92pn\nEakbNQ5fi8USvHfs0qVLufzyyxW8p7B16yZycmZRXl5B06ZNufXW2xk3bjQrVqwmJubYF35PmHA/\n7dpdSO/e/Xn++afZseN9vN4qevXqw6233g7AsmW5LF++hEAgQMOGDRk7diLNm+uNUmrGbDbTJetc\nLunW/ISf8xWR2vWbb7Kxbt065syZw0svvURi4ok/UOz1+rBao/cf9LfffsuNN97I4sWLycjIYM6c\nORw4cIAvv/ySUaNGcdVVV+HxeOjcuTOrVq1i6dKl/OMf/+CFF17A6/Vy8803c/fdd3PRRRdx5ZVX\nsnHjRhISEli1ahXfffcdw4YNC3UXRUTkF/pNC662bt3K7NmzmTt37kmDF8DpLPsth6rXfsndUd59\ndy0XXtiR1NQzKCgoISvrep555hl+//thLFv2Ju3aXcy2bZs599xWmM3xrF27jltuuQ23+9hCmKys\nnqxYsZLWrS8kEID5818lK+tqOnXqQqdOkX0Hsf+lu/8YT2NuLI23scLqDlclJSVMnz6defPmkZKS\nUtPdRI3S0hL27t3D4MH9go8lJCTQqdMlLFw4j/LycrZs2URmZncASkpKeeaZGcyZ8xwAVVVVtGnT\nFqvVytNPP88rr7zMiy/OoUWLc7n33jG0aNEyJP0SEZFfr8bh++677+J0Ohk1alTwsWnTptG4ceNa\nKSzSpKU56NTpYh55ZPrP2po3b8nWrZt4771t3HnnyH8/P42bbhrCZZd1/dnzMzJa88gj06iqquLV\nV+fzt79NZdasl+q8DyIiUjtqfJON7Oxstm3bxoIFC4L/KXhP7OKLO7N374ccOvQdAB9/vJ+nnvob\nAN27X80LLzxPixbnkppqB6Br1268886b+Hw+AoEA8+bNZceO9/j888+YMOEBqqqqiImJoXXr8wB9\nAbqISDjRvZ0NkpaWxgMPjGfcuNF4vVXEx8dz9933ApCZ2Z1nn30yuJoZoG/fgRw5coQhQwYSCARo\n3fo8Bg4cTFxcHI0bN2bIkIFYrTHEx8dzzz0PhKpbIiJSA/pKwVrwWy/WV1ZWMmDADSxY8DpJSckn\nf67PT0mVl8QYK7GW6Lw7qBajGE9jbiyNt7HCasGV1J7c3Nfo3LnLSYPXFwiw6ptCPnGV4qr0khJr\npU1KAtc0S8Ni0rSziEg4UfiG2ODB/UhNtTNlyuMnfd6qbwp5L98V3HZWeoPb15/lqNMaRUSkdil8\nQ+y115ad8jmVPj+fuEqrbfvEVUqPpo2idgpaRCQc6R07DJRUeXFVeqttc1V6Kamqvk1EROonhW8Y\nSIyxkhJb/SRFSqyVxBhNYIiIhBOFbxiItZhpk5JQbVublARNOYuIhBmdMoWJa5qlAVS72llERMKL\nwjdMWEwmrj/LQY+mjaL+c74iIuFO4RtmYi1mGlliQ12GiIj8Bjp1EhERMZjCV0RExGAKXxEREYMp\nfEVERAym8BWJYrt37yI7u3eoyxCJOgpfERERg+mjRiJh6O2332Tx4oX4fD4aNUrjwQcfYuXKFXz/\n/RHcbheff/4Z6enpTJ36N1JT7fTvfwN9+vRn/fq15Od/z4039mPYsDuP22dlZSXPP/80O3a8j9db\nRa9efbj11ttD1EORyKYzX5Ew43QW8eST03nyyedYvPgNmjRpyrx5cwHYvHkjo0aNZtmyd2jcuAkL\nFrwcfN3+/R+RkzOfBQteZ/nyJRw8mHfcfl977RW+/PJLXnllMQsWvM6mTevZvn2roX0TiRYKX5Ew\nk5pqZ/XqzaSnnwZA+/YdOHz4EAAdO/4/GjduAsDll2fy0Uf7gq/r2fM6LBYLqal22re/kP379x23\n3+3bt9C3b39iY2OJi4ujZ8/r2Lx5g0G9EokumnYWCTM+n4+5c2ezffsWfD4fZWVlnHlmMwCSkpKD\nz0tMTKSkpCS4nZSU9F9tSZSUFB+335KSUp55ZgZz5jwHQFVVFW3atK3LrohELYWvSJhZv34t27dv\nYebMHFJSUlix4g3WrFkFgMvlCj6vpKT4uMB1u39qKy52H9cGkJaWxk03DeGyy7rWcQ9ERNPOImHG\n5Sri9NPPICUlBbfbxYYNaykvLwdg374P+eGH7wHYuHE97dt3CL5u/fq1+P1+ioqOsm/fXtq163Dc\nfrt27cY777yJz+cjEAgwb95cdux4z7iOiUQRnfmKhJmsrKtZu3Y12dm9ady4CcOG3cWYMfdQVlbG\nRRddwowZ0zh4MI/TTjudUaNGB193zjnNGTbs//j++8P0759N8+Yt2L17V7C9b9+BHDlyhCFDBhII\nBGjd+jwGDhwcii6KRDxTIBAIGHGggoKSUz8pTDkciRHdv/pG4129F1+cQ0FBPmPGPPiztv79b+DB\nBx+mffsLa7RvjbmxNN7GqqvxdjgST9imaWcRCaqq8uF2llNV5Qt1KSIRTdPOIoLf7+e9DZ/zZV4h\npcUeEpJsnJORxqWZLTCb9Tu6SG1T+IpEiKFDh5+wbenSt0/62vc2fM5Huw4Ft0uLPcHtLlnn1k6B\nIhKkX2lFolxVlY8v8wqrbfsqr1BT0CJ1QOErEuXKSispLfZU21Za4qGstNLgikQin8JXJMrFJ8SS\nkGSrti0h0UZ8QqzBFYlEPoWvSJSLibFwTkZatW1nZ6QRE2MxuCKRyKcFVyLCpZktgGPXeEtLPCQk\n2jj736udRaT2KXxFBLPZTJesc7mkW3PKSiuJT4jVGa9IHVL4ikhQTIyF5NS4UJchEvF0zVdERMRg\nCl8RERGDKXxFREQMpvAVERExmMJXRETEYApfERERgyl8RUREDKbwFRERMZjCV0RExGAKXxEREYMp\nfEVERAym8BURETGYwldERMRgCl8RERGDKXwlZPbu/ZD+/W8IdRkiIoZT+IpI1KusrGTVqndCXYZE\nEWuoC5DoMm/eXFaseIPk5GS6dOkGgMfj4ZlnnmD37l2YzWZ+97vLuOuuu7FYLHzwwftMm/YIcXFx\nDBw4mOeee5q3316BzZYc4p5IJMnL+5S///1drrnm+lCXIlFCZ75imC+//ILc3NeYO/cV5s5dwGef\nHQTg9dcXkZ//AwsWvM5LLy1k3749rFu3Gp/Px5Qpkxg9ehyvvrqU7777loqK8hD3QsLFqlXvMGhQ\nXwYN6svDDz9IZWUlb7/9Jjff3J9Bg/rwpz8N4/vvj1BUdJTx4+/jwIF93HXXHaEuW6KEwlcMs3fv\nbi68sCN2eyMsFgtXX30NAO+/v41evfpgtVqx2RrQvfs17Ny5g2+//YbKyko6d74MgH79svH7/aHs\ngoSJI0cO89xzTzNz5hwWLVpGeXkFr7/+Gk8+OZ0nn3yOxYvfoEmTpsybNxe7vRHDh4+gbdt2PP/8\n3FCXLlFC085imOLiYhISEoLbiYlJALhczuCfjz2eiNPppKSk+LjH09LSjCtWwtrOnTu44IJ2pKU5\nAJg48REsFgvZ2TcTExMDQPv2HVi9+t1QlilRTOErhklMTKK0tDS47XI5AbDbG+F2u4OPFxe7sdvt\nNGzYkPLysuDjRUVHjStWwprb7SIhITG4bbPZ8Pl85OTMYvv2Lfh8PsrKyjjzzGYhrFKimaadxTDn\nn38BH330IU6nE5/Px+rVqwC49NIurFz5Fj6fj/LyclavfpfOnbvQtGkzvF4vu3fvAuDNN5dhMplC\n2QU+/ng/99wzIqQ1yKklJ6fgdruC2z/+WMqaNavYvn0LM2fmsGjRcoYOHR7CCiXa/abwzcvLIysr\ni4ULF9ZWPRLBzj23FTfe2I+hQ29h6NAhtGvXHjh2LTc9/TSGDBnIHXcM4dJLu5KZmUVsbCz33TeG\nqVMnc9ttgznzzGaYzeaQBvB5553PjBkzQ3Z8+WU6d76Mffv2cuTIYQKBAI8//igFBfmcfvoZpKQc\nC+YNG9ZSXn5sAZ/VaqWsrJRAIBDiyiVamAI1/GkrKytj+PDhnH322bRq1YpbbrnlpM8vKCipUYHh\nwOFIjOj+1Rfl5eV0796VXbt2UVERmhp2797FtGmPkJv7ZmgKCJFw/BnfsGEds2Y9i8Vipk2btgwf\n/icefHAMxcVuGjduwh133MmYMffQo8c1DBgwiDvvHIrf72P58nexWCwhrT0cxzuc1dV4OxyJJ2yr\n8TXf2NhYcnJyyMnJqekuRE5p6NAh3ND/Jq7t0ZP169dw9tnnkJiYSEWF3pjk5DIzs8jMzDrusZyc\n+cdtv/32muCf33hDi6/EODUOX6vVitWq9VpSN3yBAKu+KcRx7U3Menkus3JmkdgwgUnjJoa6NIkQ\nlb5K3J4Skm2JxFpiQ12ORBnD0jM1NR6rNbRTOXXpZNML8ust/vhb3st3YW52Lv9v9LTg4z84jn10\nJFTjnZISj8Vijsq/70jps8/vY8HeZfzju30UlhWRFm/noqbtGNK+HxZz/XmPipTxDhdGj7dh4et0\nlp36SWFK12dqV6XPz+7Dzmrbdh920qdVE4qLfjS4qmNcrjJ8Pn/U/X1H0s/4krwVbPpuW3C7oOwo\n7+ZtpKysigEZvUJY2U8iabzDQSiu+eqjRlLvlFR5cVV6q21zVXpxV1QZXJFEikpfJfsKDlTb9lHh\nASp9lQZXJNGqxuG7f/9+hgwZwhtvvMErr7zCkCFDcLlcp36hyCkkxlhJia1+UiYl1kpygxiDK5JI\n4faU4PRU/z5VVOHC7dHZphijxtPO559/PgsWLKjNWkQAiLWYaZOSwHv5P3+TbJOSgM0Sugmbjh07\nRd3HjCJJsi2RVFsKRZ6fX9awN0gh2abrrGIMTTtLvXRNszQuTU8hNdaKCUiNtXJpegrXNDP2/s6e\nKh/5zjI8VT5Djyt1I9YSSztHWwBKv3TyyVPvc2Tt5xT+4xAXpLUls9ul5Of/EOIqJRros0JSL1lM\nJq4/y0GPpo0oqfKSGGMl1sAzXp/fT+6Gz9iTV0BRsQd7ko0OGQ6yM1tiMet31nDWt+V1AGw5tAmA\n82/oxAVpbenb8jqe5qEQVibRROEr9VqsxUyjEHwGM3fDZ6zb9V1w+2ixJ7g9OCvD8Hqk9ljMFgZk\n9OKs4nSmNzhIzIZyfmyajyXj+I8ZzZnzHF999QVTpjzO119/xRNPPEZhYSGxsTGMGzeR1q3PC1EP\nJBLoV3iR/+Gp8rEnr6Datj15hZqCjhBWsxWLyYLZ9PO3wfXr1/DBB+/z178+AsDYsffRs+e1LF68\nnPvuG8uYMffi9Va/Il/kl1D4ivwPd6mHomJPtW3OkgrcpdW3SWTIy/sXOTmzmTZtBnFxcXz99Ve4\nXEVcd92NALRrdyEpKans378vxJVKONO0s8j/SE6wYU+ycbSaAE5NbEBygi0EVYlRHn/8UeLj40lK\nSgKgtLSEiooKbr65f/A5P/7443HfQS3yayl8Rf6HLcZChwzHcdd8/6NDRhq2mPpzC0KpfRMnPsJb\nby1n1qyZjBp1H2lpDho2bMhrry0LdWkSQTTtLFKN7MyWZHVqSqOkBphN0CipAVmdmpKd2TLUpUkd\na9r0TO655342blzH7t27OP30M3A4TmPjxnUAuFwuJk4cF/wuYJGa0JmvSDUsZjODszLo160F7lIP\nyQk2nfFGkeTkFEaPHsujjz7E/PmLmDx5Ko8/PpWcnFmYzWays28mLi4u1GVKGDMFAoGAEQeK5JuE\n6yboxtJ4G09jbiyNt7H0xQoiIvWQ3+OhMj8fv0cr3aV2aNpZROQEAj4fBUsWU7pnN96iIqx2Owkd\nOuIYMAiTRZchpOYUviIiJ1CwZDGudWuD296jR4Pb6YNuDlVZEgE07SwiUg2/x0Ppnt3VtpXu2aMp\naPlNFL4iItXwut14i4qqb3MW4dVNNuQ3UPiKiFTDmpyM1W6vvi3VjjU52eCKJJIofEVEqmG22Ujo\n0LHatoQOHTDbdJtRqTktuBIROQHHgEHAsWu8XmcR1lQ7CR06BB8XqSmFr4jICZgsFtIH3Uxan/54\n3W6syck645VaofAVETkFs81GbHp6qMuQCKJrviIiIgZT+IqIiBhM4SsiImIwha+IiIjBFL4iIiIG\nU/iKiIgYTOErIiJiMIWviIiIwRS+IiIiBlP4ioiIGEzhKyIiYjCFr4iIiMEUviIiIgZT+IqIiBhM\n4SsiImIwha+IiIjBFL4iIiIGU/iKiIgYTOErIiJiMIWviIiIwRS+IiIiBlP4ioiIGEzhKyIiYjCF\nr4iIiMEUviIiIgZT+IqIiBhM4SsiImIwha+IiIjBFL4iIiIGU/iKyEnNnj2TRYsWhboMkYhiDXUB\nIlK//fGPI3A4EikoKAl1KSIRQ+ErEiGOHDnMH//4ewYOHMw777xFIAATJkxm/vy5HDyYx8UX/45x\n4yayYcM6Xn75BXw+H2lpDh54YAKVlZXcddcdvP32GqzWY28LY8feyyWXdObAgf1kZLRgwIAhfPnl\nFzzxxGMUFhYSGxvDuHETad36vBD3XCT8aNpZJIK4XC7s9kYsWrScli1bMnHiWMaPn8z8+YtYt241\ne/b8k+nTH+HRR5/gtdeW0blzF6ZPn8o55zSnUaNG7Nv3IQAVFRX885+76NbtquC+/X4/Y8feR8+e\n17J48XLuu28sY8bci9frDVV3RcKWwlckgvh8PjIzswBo3rwlbdqcR0pKCsnJKTRqlMaePf+kQ4dO\nNG16JgA33NCbPXt24fV6ueKKq9i2bTMAH3zwHm3atCU1NTW476+//gqXq4jrrrsRgHbtLiQlJZX9\n+/cZ3EuR8KfwjRIff7yfe+4ZAcCUKZOYN28uAF26dCI//4dQlia1yGKxYLM1AMBsNhMXFx9sM5vN\nxMTEkpiYGHwsISGBQCCA2+36d/huAWDLlk1cdVX34/ZdWlpCRUUFN9/cn8GD+zF4cD+cziLcbrcB\nPROJLLrmGyXOO+98ZsyYGeoyJMTsdjsHDvx0plpcXIzZbA6eGZvNFg4ezGPnzh3cffc9x702Lc1B\nw4YNee21ZUaXLRJxdOYbAY4cOcyNN17Nq6/O56ab+jJoUF/27/+I0aP/TO/e1zB16mR2795Fdnbv\nk+5nzpznGDv2Xvx+v0GVi9GsVisffriHQ4e+A+Ctt5Zx0UWXBBdZXXnlVbz00guce24Gyckpx732\n9NPPwOE4jY0b1wHHri9PnDiO8vJyYzshEgF05hsh/nuhzYQJ9zNx4lhefHEhJhP06XMt7dq1P+nr\n169fwwcfvM9zz+VgNut3skjlcKQzZswExo49tlDqjDOacP/944LtV1xxFUOH3sKYMRN+9lqTycTk\nyVN5/PGp5OTMwmw2k519M3FxcUZ2QSQi1Dh8p06dyt69ezGZTIwbN4527drVZl3yK/3vQhuAlJRj\nZy6NGqVRWFh4wtfm5f2LnJzZPPvsbL2RhrEzzmjM5s0fBLdvu+2O49pzc98M/vmKK66iOq1atWbb\ntl3HPTZ+/KTg53zPOutsZs58oRarFolONTrF2blzJ19//TW5ublMmTKFKVOm1HZd8iudaqFNu3YX\nnvC1jz/+KCYTJCUl1XmdEt78Hg+V+fn4PZ5QlyJyQg899CB9+15Hly6d+PTTf53weUeOHKZbt0sM\nrOwnNTrzff/998nKOnaW1aJFC9xuN6WlpSQkJNRqcWKMiRMf4a23ljNr1kxGjbov1OVIPRTw+chf\n/Cqle3bjLSrCareT0KEjjgGDMFksoS5P5Djr1q1m0aLlNGnSNNSlnFCNwrewsJC2bdsGt+12OwUF\nBScN39TUeKzWyP1H6nAknvpJdcTjaXhcDQ0b2mjQICa4bbGYSUmJx2Ix43Ak0qBBDA0b2oLt7du3\n4eKLH+KGG26gV69rueSS0Pwm+GuEcryj0Rc5L+Fatza47T16FNe6tcQ1iKX5sNtDWFnk0s94zQwZ\nMgS/38/o0XdTWlrKs88+y4UXXsjEiRPZtWsXfr+fVq1a8dhjj2G3H3vvXLp0KfPnz6e4uJjRo0dz\n/fXX13mdtbLgKhAInPI5TmdZbRyqXgr1fW+Lin4ECNbw448eKiqqgts+nx+Xqwyfz09BQQkVFVX8\n+KMn2H70aCnp6Q25994xPPDAGObPX0R8fMPQdOYXCPV4Rxu/x0PRzp3VthW8/wENr+mF2WYzuKrI\npp/xmpsx43m6dOnEU0/N4q677sDlKmflyrV8+eXXLFiwBIC5c2ezadN7NGt2Fn6/n6qqKl566TU2\nblzHE0/M4JJLutVKLSf7BapG4Zuenn7cAp78/HwcDkdNdiW14JcutPnP/8ePnxRs++/FNV26dKNL\nl9r5oZPI4XW78RRUv2DP6yzC63YTm55ucFUiv1xKSgpfffUFW7Zs5OKLOzNs2J3AsWu+gUCA3r17\nU1rqJSOjNQUF+YbUVKMFV5dddhmrV68G4MCBA6Snp+t6bxjyVPnId5bhqfKFuhSpx6zJydgcadW3\npdqxJicbXJHIr3PeeeczatQuSq3kAAAO8UlEQVRoli7NpVevq5k0aTwlJcdmFiwWS/BTHmaz2bD7\nHNTozLdjx460bduWQYMGYTKZmDhxYm3XJXXI5/eTu+Ez9uQVUFTswZ5ko0OGg+zMllj0GV/5H2ab\nDfvFF3PknZU/a0vo0EFTzhIWrrwyiyuvzKK42M2jjz7Ea6+9Qq9efUJWT42v+d53n1bFhqvcDZ+x\nbtd3we2jxZ7g9uCsjFCVJfXYObf/H+UVlZTu2YPXWYQ11U5Chw44BgwKdWkip7Ry5QoKCvK57bY7\nSEpKplmzszGZTCGtSXe4ijKeKh978gqqbduTV0i/bi2wxUTuqnSpGZPFQvqgm0nr0x+v2401OVln\nvBI2unbtxqOPPsSgQX2wWCw0bXom48dP4scffwxZTabAL1mqXAsieeVeOK1MzHeWMXbODqr7Szeb\nYOoffkd6anw1rfVHOI13pNCYG0vjbay6Gu+TrXbWBb4ok5xgw55U/RlLamIDkhN0NiMi0cXvq6TK\nU4TfX2XYMTXtHGVsMRY6ZDiOu+b7Hx0y0jTlLCJRIxDw4zy0hu8/OUhlhRNLTDJxKa1IbdIDk6lu\nz00VvlEoO/PYFy/sySvEWVJBamIDOmSkBR8XEYkGzkNrKC346QYyvip3cNvetGedHlvhG4UsZjOD\nszLo160F7lIPyQk2nfGKSFTx+6sod31abVu5Kw9/46swm2Pq7Pi65hvFbDEW0lPjFbwiEnV8VSX4\nqtwnaHPjq6rbBW8KXxERiTqWmEQsMdXfnc0Sk4wlpm6/2ELhKyIiUcdsjiEupVW1bXEpGXU65Qy6\n5isiIlEqtUkPACpLDlJZ4fr3aueM4ON1SeErIiJRyWQyY2/ak0b2G/n+yBEsMYl1fsb7HwpfERGJ\namZLLDE2u7HHNPRoIiIiovAVERExmsJXRETEYGEbvrNnz+TNN5eGugwREZFfLWwXXP3xjyNCXYKI\niEiN1PmZ73fffUeXLl149dX53HRTXwYN6sv+/R8xevSf6d37GqZOnczu3bvIzu4dfM1/b3/xxWcM\nH/57brllIIMG9WHZslwApkyZxLx5cwH4178+4fbbb2HQoL6MGPEHDh8+VNfdEhERqTFDpp2dTid2\neyMWLVpOy5YtmThxLOPHT2b+/EWsW7eaw4d//vV2//HSSzn07t2PhQtfZ/bsl9m1ayeVlZXHPWfS\npHEMG3Ynixcv5/LLr+DJJ6fXdZdERERqzJBpZ6/XS2ZmFgDNmx/72rqUlBQAGjVKo7Cw8ISvTU21\ns2nTepo3b0lGRiseffSJ49q/+eZrXC4XnTtfBkDfvgO58cZ+ddENERGRWmHIma/FYsFma3DsgGYz\ncXHxPxVgNtOu3YUnfO2dd46kefOW/PWvY+jb9zqWL19yXLvb7SIhISG4bbVasdlstdwDERGR2lMv\nFlzt2/chfr8/uF1S8tNXOcXHxzN8+J8YPvxPfPLJAe699246dbo42J6cnEJxcTF+vx+z2YzX66Wg\nIJ8zzmhsaB9ERER+qXrxUaNGjdI4erQQp7MIn8/H2rWrgm333/8XvvjicwCaN29Bw4YJmEymYPuZ\nZzYjPT2dzZs3APDOO28yffoUYzsgIiLyK9SLM98mTZpy7bW9+P3vb+a0006nZ8/rOHgwD4D+/bOZ\nPHkCXm8VAH369OfMM5sFX2symXjoocd4+OEHmTPnORo1SmPcuIkh6YeIiMgvYQoEAgEjDlRQUHLq\nJ4UphyMxovtX32i8jacxN5bG21h1Nd4OR+IJ2+rFtPNv4anyke8sw1PlC3UpIiIiv0i9mHauCZ/f\nT+6Gz9iTV0BRsQd7ko0OGQ6yM1tiMYf97xQiIhLBwjZ8czd8xrpdP92c42ixJ7g9OCsjVGWJiIic\nUlieInqqfOzJK6i2bU9eoaagRUSkXgvL8HWXeigq9lTb5iypwF1afZuIiEh9EJbhm5xgw55U/V2s\nUhMbkJygO1yJiEj9FZbha4ux0CHDUW1bh4w0bDEWgysSERH55cJ2wVV25rEvaNiTV4izpILUxAZ0\nyEgLPi4iIlJfhW34WsxmBmdl0K9bC9ylHpITbDrjFRGRsBC24fsfthgL6anxp36iiIhIPRGW13xF\nRETCmcJXRETEYApfERERgyl8RUREDKbwFRERMZjCV0RExGAKXxEREYMpfEVERAym8BURETGYwldE\nRMRgCl8RERGDKXxFREQMpvAVERExmMJXRETEYApfERERgyl8RUREDKbwFRERMZjCV0RExGAKXxER\nEYMpfEVERAym8BURETGYwldERMRgNQ7fnTt30rlzZzZu3Fib9YiIiES8GoXvN998w8svv0zHjh1r\nux4REZGIV6PwdTgczJw5k8TExNquR0REJOJZa/KiuLi42q5DREQkapwyfJcsWcKSJUuOe2zkyJF0\n7dr1Vx0oNTUeq9Xy66oLIw6HZgGMpPE2nsbcWBpvYxk93qcM3wEDBjBgwIDffCCns+w376O+cjgS\nKSgoCXUZUUPjbTyNubE03saqq/E+WaDro0YiIiIGq1H4btq0iSFDhrB161ZmzJjB7bffXtt1iYiI\nRKwaLbi64ooruOKKK2q5FBERkeigaWcRERGDKXxFREQMpvAVERExmMJXRETEYApfERERgyl8RURE\nDKbwFRERMZjCV0RExGAKXxEREYMpfEVERAym8BURETGYwldERMRgCl8RERGDKXxFREQMpvAVEREx\nmMJXRETEYApfERERgyl8RUREDKbwFRERMZjCV0RExGAKXxEREYMpfEVERAym8BURETGYwldERMRg\nCl8RERGDKXxFREQMpvAVERExmMJXRETEYApfOandu3eRnd27Vvb17rtv8+c/31Ur+xIRCWcKXxER\nEYNZQ12AhIeZM59i27YtmM0mxo79KxkZrXnmmSfYvXsXZrOZ3/3uMu66624sFguffXaQJ554FLfb\nTWysjTvvHMkll3Q+bn8//ljKH/94O7ff/geuvDIrRL0SEQkNha+c0vffH6F16zaMGDGKFSveYMaM\naWRm9iA//wcWLHgdn8/LiBF/YN261XTv3pNJk8bxf/83lO7de/Kvf33MX/4ygmXL3g7uz+/3M3ny\nBHr0uEbBKyJRSdPOckqxsbFkZnYHIDOzOwcP5rF58wZ69eqD1WrFZmtA9+7XsHPnDo4cOczRo0fJ\nyroagNatz+P000/nk08+Du5v9uyZpKSkMmTI70PSHxGRUFP4yiklJSVjNh/7UWnYsCFwbNo4MTEp\n+JzExEScTidOp5OEhERMJtN/tSXhdBYB8OmnH7NsWS4OR7qBPRARqV8UvnJKJSUlP/tzUlIybrc7\n+HhxsRu73Y7dbqekxE0gEAi2ud1u7PZGAKSlOXjllVxWrHiDTz45YFAPRETqF4WvnJLHU8HmzRsB\n2LRpPW3anEe3bleycuVb+Hw+ysvLWb36XTp37sIZZzTG4Uhn/fo1AHz00V6Kio7Spk1bABo1ctCk\nSVNGjvwLU6ZMwuPxhKxfIiKhogVXckrNmp3FgQP7mDNnJmazmfHjJ9G8eUsOHz7EkCEDMZlMXHll\nFpmZWZhMJiZPnsrjjz/Kyy/n0KBBHA8//BhxcXHH7bNHj2vYtGkDL7zwPCNH/iVEPRMRCQ1T4L/n\nB+tQQUHJqZ8UphyOxIjuX22p9FXi9pSQbEsk1hJb4/1ovI2nMTeWxttYdTXeDkfiCdt05it1zuf3\nsfyzlewrOIDT4yLVlkI7R1v6trwOi9kS6vJERAyn8JU6t/yzlWz6bltwu8jjDG4PyOgVqrJEREJG\nC66kTlX6KtlXUP2q5o8KD1DpqzS4IhGR0FP4Sp1ye0pwelzVthVVuHB7dF1LRKKPwlfqVLItkVRb\nSrVt9gYpJNtOvCBBRCRSKXylTsVaYmnnaFtt2wVpbX/TqmcRkXClBVdS5/q2vA44do23qMKFvUEK\nF6S1DT4uIhJtFL5S5yxmCwMyenFji5618jlfEZFwp/AVw8RaYnHENwp1GSIiIadrviIiIgZT+IqI\niBhM4SsiImIwha+IiIjBFL4iIiIGU/iKiIgYTOErIiJiMIWviIiIwUyBQCAQ6iJERESiic58RURE\nDKbwFRERMZjCV0RExGAKXxEREYMpfEVERAym8BURETGYwrcWeL1eHnjgAW666SYGDhzIrl27Ql1S\nxNu5cyedO3dm48aNoS4lok2dOpXs7GwGDRrEvn37Ql1OVMjLyyMrK4uFCxeGupSoMH36dLKzs+nX\nrx9r1qwx7LhWw44Uwd566y3i4uJYtGgRBw8eZOzYsSxdujTUZUWsb775hpdffpmOHTuGupSItnPn\nTr7++mtyc3P5/PPPGTduHLm5uaEuK6KVlZXx8MMP07lz51CXEhV27NjBwYMHyc3Nxel00qdPH3r0\n6GHIsXXmWwt69erF2LFjAbDb7bhcrhBXFNkcDgczZ84kMTEx1KVEtPfff5+srCwAWrRogdvtprS0\nNMRVRbbY2FhycnJIT08PdSlR4aKLLuLpp58GICkpifLycnw+nyHHVvjWgpiYGGw2GwDz58/n+uuv\nD3FFkS0uLg6LxRLqMiJeYWEhqampwW273U5BQUEIK4p8VquVBg0ahLqMqGGxWIiPjwdg6dKlXH75\n5Ya9t2ja+VdasmQJS5YsOe6xkSNH0rVrV1599VUOHDjA7NmzQ1Rd5DnZeIuxdCdaiVTr1q1j6dKl\nvPTSS4YdU+H7Kw0YMIABAwb87PElS5awYcMGnn/+eWJiYkJQWWQ60XhL3UtPT6ewsDC4nZ+fj8Ph\nCGFFIrVv69atzJ49m7lz5xp6KUvTzrXg22+/ZfHixcycOTM4/SwS7i677DJWr14NwIEDB0hPTych\nISHEVYnUnpKSEqZPn86cOXNISUkx9Nj6VqNaMGPGDFauXEnjxo2Dj7344ovExsaGsKrItWnTJl58\n8UW++OIL7HY7DofD0OmiaPK3v/2NXbt2YTKZmDhxIq1btw51SRFt//79TJs2jUOHDmG1WjnttNN4\n9tlnDQ+GaJGbm8uzzz7LOeecE3xs2rRpx72X1xWFr4iIiME07SwiImIwha+IiIjBFL4iIiIGU/iK\niIgYTOErIiJiMIWviIiIwRS+IiIiBlP4ioiIGOz/A0vBwFbJIDthAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x396 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "'''\n",
    "  code by Tae Hwan Jung(Jeff Jung) @graykode\n",
    "'''\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "dtype = torch.FloatTensor\n",
    "\n",
    "# 3 Words Sentence\n",
    "sentences = [ \"i like dog\", \"i like cat\", \"i like animal\",\n",
    "              \"dog cat animal\", \"apple cat dog like\", \"dog fish milk like\",\n",
    "              \"dog cat eyes like\", \"i like apple\", \"apple i hate\",\n",
    "              \"apple i movie book music like\", \"cat dog hate\", \"cat dog like\"]\n",
    "\n",
    "word_sequence = \" \".join(sentences).split()\n",
    "word_list = \" \".join(sentences).split()\n",
    "word_list = list(set(word_list))\n",
    "word_dict = {w: i for i, w in enumerate(word_list)}\n",
    "\n",
    "# Word2Vec Parameter\n",
    "batch_size = 20  # To show 2 dim embedding graph\n",
    "embedding_size = 2  # To show 2 dim embedding graph\n",
    "voc_size = len(word_list)\n",
    "\n",
    "def random_batch(data, size):\n",
    "    random_inputs = []\n",
    "    random_labels = []\n",
    "    random_index = np.random.choice(range(len(data)), size, replace=False)\n",
    "\n",
    "    for i in random_index:\n",
    "        random_inputs.append(np.eye(voc_size)[data[i][0]])  # target\n",
    "        random_labels.append(data[i][1])  # context word\n",
    "\n",
    "    return random_inputs, random_labels\n",
    "\n",
    "# Make skip gram of one size window\n",
    "skip_grams = []\n",
    "for i in range(1, len(word_sequence) - 1):\n",
    "    target = word_dict[word_sequence[i]]\n",
    "    context = [word_dict[word_sequence[i - 1]], word_dict[word_sequence[i + 1]]]\n",
    "\n",
    "    for w in context:\n",
    "        skip_grams.append([target, w])\n",
    "\n",
    "# Model\n",
    "class Word2Vec(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Word2Vec, self).__init__()\n",
    "\n",
    "        # W and WT is not Traspose relationship\n",
    "        self.W = nn.Parameter(-2 * torch.rand(voc_size, embedding_size) + 1).type(dtype) # voc_size > embedding_size Weight\n",
    "        self.WT = nn.Parameter(-2 * torch.rand(embedding_size, voc_size) + 1).type(dtype) # embedding_size > voc_size Weight\n",
    "\n",
    "    def forward(self, X):\n",
    "        # X : [batch_size, voc_size]\n",
    "        hidden_layer = torch.matmul(X, self.W) # hidden_layer : [batch_size, embedding_size]\n",
    "        output_layer = torch.matmul(hidden_layer, self.WT) # output_layer : [batch_size, voc_size]\n",
    "        return output_layer\n",
    "\n",
    "model = Word2Vec()\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "# Training\n",
    "for epoch in range(5000):\n",
    "\n",
    "    input_batch, target_batch = random_batch(skip_grams, batch_size)\n",
    "\n",
    "    input_batch = Variable(torch.Tensor(input_batch))\n",
    "    target_batch = Variable(torch.LongTensor(target_batch))\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "    output = model(input_batch)\n",
    "\n",
    "    # output : [batch_size, voc_size], target_batch : [batch_size] (LongTensor, not one-hot)\n",
    "    loss = criterion(output, target_batch)\n",
    "    if (epoch + 1)%1000 == 0:\n",
    "        print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n",
    "\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "for i, label in enumerate(word_list):\n",
    "    W, WT = model.parameters()\n",
    "    x,y = float(W[i][0]), float(W[i][1])\n",
    "    plt.scatter(x, y)\n",
    "    plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom')\n",
    "plt.show()\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "Word2Vec-Torch(Softmax).ipynb",
   "version": "0.3.2",
   "provenance": [],
   "collapsed_sections": []
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "accelerator": "GPU"
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
